{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keras Backend\n",
    "\n",
    "In this notebook we will be using the [Keras backend module](http://keras.io/backend/), which provides an abstraction over both Theano and Tensorflow."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try to re-implement the Logistic Regression Model using the `keras.backend` APIs.\n",
    "\n",
    "The following code will look like very similar to what we would write in Theano or Tensorflow (with the *only difference* that it may run on both the two backends)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras.backend as K\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from kaggle_data import load_data, preprocess_data, preprocess_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9 classes\n",
      "93 features\n"
     ]
    }
   ],
   "source": [
    "X_train, labels = load_data('data/kaggle_ottogroup/train.csv', train=True)\n",
    "X_train, scaler = preprocess_data(X_train)\n",
    "Y_train, encoder = preprocess_labels(labels)\n",
    "\n",
    "X_test, ids = load_data('data/kaggle_ottogroup/test.csv', train=False)\n",
    "\n",
    "X_test, _ = preprocess_data(X_test, scaler)\n",
    "\n",
    "nb_classes = Y_train.shape[1]\n",
    "print(nb_classes, 'classes')\n",
    "\n",
    "dims = X_train.shape[1]\n",
    "print(dims, 'features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feats = dims\n",
    "training_steps = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = K.placeholder(dtype=\"float\", shape=X_train.shape) \n",
    "target = K.placeholder(dtype=\"float\", shape=Y_train.shape)\n",
    "\n",
    "# Set model weights\n",
    "W = K.variable(np.random.rand(dims, nb_classes))\n",
    "b = K.variable(np.random.rand(nb_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define model and loss\n",
    "y = K.dot(x, W) + b\n",
    "loss = K.categorical_crossentropy(y, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "activation = K.softmax(y) # Softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = K.constant(0.01)\n",
    "grads = K.gradients(loss, [W,b])\n",
    "updates = [(W, W-lr*grads[0]), (b, b-lr*grads[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = K.function(inputs=[x, target], outputs=[loss], updates=updates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: [ -498.89337158 -1137.16247559  -410.74249268 ...,    61.32589722\n",
      " -1150.31567383  -178.36450195]\n",
      "Loss: [-26216156. -27188296. -29494026. ..., -25988334. -26679148. -28535620.]\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "loss_history = []\n",
    "for epoch in range(training_steps):\n",
    "    current_loss = train([X_train, Y_train])[0]\n",
    "    loss_history.append(current_loss)\n",
    "    if epoch % 20 == 0:\n",
    "        print(\"Loss: {}\".format(current_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss_history = [np.mean(lh) for lh in loss_history]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAERCAYAAABsNEDqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt0VPXd7/H3t6ASlWNEESERxWo5hWQSSkS5PVoRQURB\nHqyK1nvR1eLlrIpg4VgvywUtLdrH1kc4akVF0VJRHqGCoBVZ1UIC4RIRuYg1gRaKDQWRSvB7/pjJ\nEGgSZpLM7Ll8XmtlZfaePXt/N6N82fu3P3ubuyMiIhKrbwRdgIiIpBc1DhERiYsah4iIxEWNQ0RE\n4qLGISIicVHjEBGRuGRs4zCzZ8xsu5mtjWHZR82sPPLzsZlVJ6NGEZF0ZJma4zCz/wD2AM+5e0Ec\nn7sD6OHuNyesOBGRNJaxRxzuvgT4vO48M/ummb1pZmVm9p6Z/e96PnoN8FJSihQRSUOtgy4gyaYD\nt7v7BjM7F3gCuLD2TTM7HegCvB1QfSIiKS9rGoeZHQ/0AX5nZrWzjzlssauB2e5+IJm1iYikk6xp\nHIRPy1W7e3Ejy1wN/ChJ9YiIpKWMHeM4nLv/E/jEzK4EsLCi2vcj4x0nAu8HVKKISFrI2MZhZi8R\nbgJdzazSzG4BrgVuMbNVQAUwrM5HrgZmeaZeZiYi0kIy9nJcERFJjIw94hARkcTIyMHxk08+2c84\n44ygyxARSRtlZWV/d/f2sSybkY3jjDPOoLS0NOgyRETShpl9GuuyOlUlIiJxUeMQEZG4qHGIiEhc\nMnKMQzLX/v37qaysZN++fUGXIpKW2rRpQ35+PkcddVST16HGIWmlsrKStm3bcsYZZ1DnnmMiEgN3\nZ+fOnVRWVtKlS5cmryfQU1VmNtjM1pvZRjMbX8/7Zmb/FXl/tZl9J1G1vLayir6T36bL+Hn0nfw2\nr62sStSmpBn27dvHSSedpKYh0gRmxkknndTsI/bAjjjMrBXwG2AgUAksN7O57v5hncUuAc6O/JwL\n/Hfkd4t6bWUV9726hi/3h2+KW1X9Jfe9ugaA4T3yWnpz0kxqGiJN1xL//wR5xNEL2Ojum939K2AW\nh947isj0cx72AZBrZh1bupApC9ZHm0atL/cfYMqC9S29KRGRtBdk48gDPqszXRmZF+8yAJjZaDMr\nNbPSHTt2xFXI1uov45ov2e34449v9jq2bt3KyJEjG3y/urqaJ554IublD3fjjTfSpUsXiouLKSoq\nYvHixc2qt6U9+eSTPPfcc81ax5o1ayguLqa4uJh27dpF9/eiiy6Kaz2DBg1i9+7djS4zYcIE3nnn\nneaUe0T9+vWjvLw8odtoKRkzOO7u0wk/4Y+SkpK47tzYKTeHqnqaRKfcnJYpTgLz2soqpixYz9bq\nL+mUm8PYQV1T4vRjp06dmD17doPv1zaOH/7whzEtX58pU6YwcuRI3nnnHUaPHs2GDRuaVTNATU0N\nrVs3/6+N22+/vdnrKCwsjP5Fe+ONNzJ06NB6m+uRal6wYMERt/XII480vdAMFOQRRxVwWp3p/Mi8\neJdptrGDupJzVKtD5uUc1Yqxg7q29KYkiWrHrqqqv8Q5OHaViAsftmzZwoUXXkgoFGLAgAH85S9/\nAWDTpk2cd955FBYWMnHixOjRypYtWygoKACgoqKCXr16UVxcTCgUYsOGDYwfP55NmzZRXFzM2LFj\nD1n+wIED3HPPPRQUFBAKhXj88ccbra13795UVR3c57KyMs4//3x69uzJoEGD2LZtGwDLly8nFApF\nt1m7vWeffZbLL7+cCy+8kAEDBgDhpnTOOecQCoX46U9/CsAXX3zBpZdeSlFREQUFBbz88ssAjB8/\nnm7duhEKhbjnnnsAeOCBB/jFL34BQHl5Oeeddx6hUIgrrriCf/zjHwBccMEFjBs3jl69evGtb32L\n9957L+bvY9GiRVxwwQUMHTqUwsJCAC677DJ69uxJ9+7deeqpp6LL5ufnU11dzcaNGykoKOCWW26h\ne/fuXHLJJdFB5Ouuu47XXnstuvwDDzxAjx49CIVCfPzxxwBs376dAQMG0L17d2677Tby8vKorq4+\npK6amhpyc3O588476d69OwMHDmTnzp3R92fNmkWvXr3o2rUrf/rTn4Dwf0P9+/enR48e9OzZkz//\n+c8AVFVV0a9fP4qLiykoKIgu/4c//IHevXvzne98h6uuuoovvvgi5j+3WAXZOJYDZ5tZFzM7mvDz\nMOYetsxc4PrI1VXnAbvcfVtLFzK8Rx6TRhSSl5uDAXm5OUwaUZgS/zKVpkvm2NUdd9zBDTfcwOrV\nq7n22mu58847Abjrrru46667WLNmDfn5+fV+9sknn+Suu+6ivLyc0tJS8vPzmTx5Mt/85jcpLy9n\nypQphyw/ffp0tmzZQnl5eXR7jXnzzTcZPnw4EM7B3HHHHcyePZuysjJuvvlmJkyYAMBNN93EtGnT\nKC8vp1WrQ/8htWLFCmbPns27777LwoUL2bBhA8uWLaO8vJyysjKWLFnCm2++SadOnVi1ahVr165l\n8ODB7Ny5kzlz5lBRUcHq1auZOHHiv9V3/fXX87Of/YzVq1dTWFjIgw8+GH2vpqaGZcuW8dhjjx0y\nPxalpaU88cQTrFu3DoAZM2ZQVlbG8uXLmTp1arRB1bV+/XruvvtuKioqyMnJiTaLw3Xo0IGVK1dy\n6623MnXqVADuv/9+Bg8eTEVFBZdddhlbt26t97O7du2ib9++VFRU0Lt3bx5++OHoe+7OsmXLmDJl\nCg899BAAHTt25K233mLlypXMnDkz+t/WCy+8wGWXXUZ5eTmrVq0iFAqxfft2Jk+ezOLFi1mxYgWh\nUIhf/epXcf25xSKwU1XuXmNmY4AFQCvgGXevMLPbI+8/CcwHhgAbgb3ATYmqZ3iPPDWKDJPMsav3\n33+fV199FYDvf//73HvvvdH5tX/5jBo1Kvov7rp69+7NI488QmVlJSNGjODss89udFuLFi3i9ttv\nj55+adeuXb3LjR07lp/85CdUVlby/vvhB1uuX7+etWvXMnDgQCB89NKxY0eqq6vZvXs3vXv3jtb6\nxhtvRNc1cODA6HYWLlzIwoUL6dGjBwB79uxhw4YN9O/fnx//+MeMGzeOoUOH0r9/f2pqamjTpg23\n3HILQ4cOZejQoYfUuGvXLqqrqzn//PMBuOGGG7jyyiuj748YMQKAnj17smXLlkb/XA7Xu3dvOnfu\nHJ1+9NFHmTs3/G/TyspKNm3aRElJySGfOeuss6JHKI1ts25d8+fPB2Dp0qXRJjx06FDatm1b72db\nt24d3cfrrruOUaNG1bve2m3/61//YsyYMaxatYrWrVuzadMmAM455xxuu+029u3bx/DhwykqKmLR\nokV8+OGH9OnTB4CvvvqKfv36xfCnFZ9AxzjcfT7h5lB33pN1Xjsp+gzwVD13Lgely9jVqFGjOPfc\nc5k3bx5Dhgxh2rRpnHnmmc1eb+0Yx+OPP87NN99MWVkZ7k737t2jjaTW4adUDnfcccdFX7s79913\nH7fddtu/LbdixQrmz5/PxIkTGTBgAPfffz/Lli1j8eLFzJ49m1//+te8/fbbMe/DMcccA0CrVq2o\nqamJ+XOH17xo0SKWLFnCBx98QE5ODv369as3y1C7vSNtszl1Ha7u5bH1rfeXv/wlp512Gi+88AL7\n9++Pnu688MIL+eMf/8i8efO4/vrruffeezn22GMZPHgwzz//fLNqOhLdq6oJknnuXJoumWNXffr0\nYdasWQDMnDmT/v37A3Deeefx+9//HiD6/uE2b97MmWeeyZ133smwYcNYvXo1bdu2bfBKn4EDBzJt\n2rToXyyff/55o7WNGTOGr7/+mgULFtC1a1d27NgRbRz79++noqKC3Nxc2rZtGz1/3lCtEL4K6Zln\nnmHPnj1A+Fz79u3b2bp1K8ceeyzXXXcdY8eOZcWKFezZs4ddu3YxZMgQHn30UVatWnXIuk444QRO\nPPHE6PjF888/Hz36aEm7du2iXbt25OTkUFFRwfLly1t8G3379uWVV14BYP78+Q1+fzU1NdGj0xdf\nfPGIRwS7du2iY8eOmBkzZsyg9qmtn376KaeeeiqjR4/mpptuYuXKlfTp04d3332XzZs3A+Fxp5a4\nKOJwahxNoNxHekjU2NXevXvJz8+P/kydOpXHH3+c3/72t4RCIZ5//vnoeeXHHnuMqVOnEgqF2Lhx\nIyeccMK/re+VV16hoKCA4uJi1q5dy/XXX89JJ51E3759KSgoYOzYsYcsf+utt9K5c2dCoRBFRUW8\n+OKLjdZrZkycOJGf//znHH300cyePZtx48ZRVFREcXFxdFD16aef5gc/+AHFxcV88cUX9dYKcPHF\nFzNq1Ch69+5NYWEhI0eOZPfu3axZsyY6yP/ggw8yceJEdu/ezdChQwmFQvTr1y86HlDXjBkzGDt2\nLKFQiPLycu6///6Yvod4XHrppezdu5du3boxceJEzj23xXPEPPjgg8ybN4+CggLmzp3LKaeccshR\nT60TTjiB9957j+7du7N06dJ6x33qGjNmDE899RRFRUV88skn0aOSxYsXU1RURI8ePXj11Ve54447\n6NChA08//TRXXXUVRUVF9OnTJzp435Iy8pnjJSUlnsgHOXUZP4/6/tQM+GTypQnbrsC6dev49re/\nHXQZMdu7dy85OTmYGbNmzeKll17i9ddfD7qseu3Zsyd6GmTy5Mls27YtIQOrmWrfvn20bt2a1q1b\ns3TpUu6+++5/e6BcTU0NJ5988hFPDSZaff8fmVmZu5c08JFDZEyOI5nS5dy5BK+srIwxY8bg7uTm\n5vLMM88EXVKD5s2bx6RJk6ipqeH000/n2WefDbqktLJlyxauueYaDhw4wDHHHMO0adOCLilhdMTR\nBIff2wrC5851CW/ipdsRh0gq0hFHAGqbg66qCoa760aHIk3UEgcLahxNpNxHMNq0acPOnTt1a3WR\nJqh9HkebNm2atR41Dkkr+fn5VFZWEu+NLEUkrPYJgM2hxpFECg0231FHHdWsJ5eJSPOpcSSJHhYl\nIplCAcAkUWhQRDKFGkeS6GFRIpIp1DiSpKFwoEKDIpJu1DiSRA+LEpFMocHxJFFoUEQyhRpHEik0\nKCKZQI0jxSn7ISKpRo0jhSn7ISKpSIPjKUzZDxFJRWocKUzZDxFJRWocKUzZDxFJRYE0DjNrZ2Zv\nmdmGyO8TG1hui5mtMbNyM0vck5lSlLIfIpKKgjriGA8sdvezgcWR6YZ8192LY30yVSYZ3iOPSSMK\nycvNwYC83Bw9ZVBEAhfUVVXDgAsir2cAfwTGBVRLSlP2Q0RSTVCNo4O7b4u8/ivQoYHlHFhkZgeA\nae4+PSnVpTllP0QkkRLWOMxsEXBqPW9NqDvh7m5mDT0Et5+7V5nZKcBbZvaRuy9pYHujgdEAnTt3\nbkbl6U3ZDxFJtISNcbj7Re5eUM/P68DfzKwjQOT39gbWURX5vR2YA/RqZHvT3b3E3Uvat2/f8juU\nJpT9EJFEC2pwfC5wQ+T1DcDrhy9gZseZWdva18DFwNqkVZimlP0QkUQLqnFMBgaa2Qbgosg0ZtbJ\nzOZHlukALDWzVcAyYJ67vxlItWlE2Q8RSbRABsfdfScwoJ75W4EhkdebgaIkl5b2xg7qesgYByj7\nISItSzc5zDB67oeIJJoaRwZS9kNEEkmNQ5T7EJG4qHFkOeU+RCReujtullPuQ0TipcaR5ZT7EJF4\nqXFkOeU+RCReahxZTs/8EJF4aXA8yyn3ISLxUuMQ5T5EJC46VSUiInHREYc0iUKDItlLjUPiptCg\nSHbTqSqJm0KDItlNjUPiptCgSHZT45C4KTQokt3UOCRuCg2KZDcNjkvcFBoUyW5qHNIkCg2KZC81\nDkkaZT9EMoMahySFsh8imUOD45IUyn6IZI5AGoeZXWlmFWb2tZmVNLLcYDNbb2YbzWx8MmuUlqXs\nh0jmCOqIYy0wAljS0AJm1gr4DXAJ0A24xsy6Jac8aWnKfohkjkAah7uvc/cjnaPoBWx0983u/hUw\nCxiW+OokEZT9EMkcqTzGkQd8Vme6MjKvXmY22sxKzax0x44dCS9O4jO8Rx6TRhSSl5uDAXm5OUwa\nUaiBcZE0lLCrqsxsEXBqPW9NcPfXW3p77j4dmA5QUlLiLb1+aT5lP0QyQ8Iah7tf1MxVVAGn1ZnO\nj8yTLKHch0hqSuUcx3LgbDPrQrhhXA2MCrYkSRblPkRSV1CX415hZpVAb2CemS2IzO9kZvMB3L0G\nGAMsANYBr7h7RRD1SvIp9yGSugI54nD3OcCceuZvBYbUmZ4PzE9iaZIilPsQSV2pfFWVZDHlPkRS\nlxqHpCTlPkRSVyoPjksW0zM/RFKXGoekLOU+RFKTGodkFGU/RBJPjUMyhrIfIsmhwXHJGMp+iCSH\nGodkDGU/RJJDjUMyhrIfIsmhxiEZQ9kPkeTQ4LhkDGU/RJJDjUMyirIfIomnU1UiIhIXHXFI1lNo\nUCQ+ahyS1RQaFImfTlVJVlNoUCR+ahyS1RQaFImfGodkNYUGReKnxiFZTaFBkfhpcFyymkKDIvFT\n45Csp9CgSHwCaRxmdiXwAPBtoJe7lzaw3BZgN3AAqHH3kmTVKNIQ5T4k2wV1xLEWGAFMi2HZ77r7\n3xNcj0hMlPsQCWhw3N3XubsulJe0o9yHSIyNI3Jq6YjzEsCBRWZWZmajG1vQzEabWamZle7YsSMJ\npUk2Uu5DJPYjjvtinBdlZovMbG09P8PiqK+fuxcDlwA/MrP/aGhBd5/u7iXuXtK+ffs4NiESO+U+\nRI4wxmFmlwBDgDwz+686b/0voKaxz7r7Rc0tzt2rIr+3m9kcoBewpLnrFWmqsYO6HjLGAcp9SPY5\n0uD4VqAUuBwoqzN/N/B/ElUUgJkdB3zD3XdHXl8MPJTIbYociXIfImDufuSFzI5y9/2R1ycCp7n7\n6iZv1OwK4HGgPVANlLv7IDPrBDzl7kPM7ExgTuQjrYEX3f2RWNZfUlLipaX1XuErIiL1MLOyWCMP\nsV6O+5aZXR5ZvgzYbmZ/cvcmHXW4+xwONoW687cSPjWGu28GipqyfpFUo+yHZJJYB8dPcPd/Es5e\nPOfu5wIDEleWSOaozX5UVX+JczD78drKqqBLE2mSWBtHazPrCHwPeCOB9YhkHGU/JNPE2jgeAhYA\nm9x9eWT8YUPiyhLJHMp+SKaJaYzD3X8H/K7O9GbgPxNVlEgm6ZSbQ1U9TULZD0lXsSbH881sjplt\nj/z83szyE12cSCbQMz8k08R6quq3wFygU+TnfyLzROQIhvfIY9KIQvJyczAgLzeHSSMKdVWVpK1Y\ncxzlkVt/NDovVSjHISISn0TkOHaa2XXAS5Hpa4CdTSlORGKj7IekqlhPVd1M+FLcvwLbgJHAjQmq\nSSTrKfshqSyey3FvcPf27n4K4UbyYOLKEsluyn5IKou1cYTc/R+1E+7+OdAjMSWJiLIfkspibRzf\niNzcEAAza0dwj50VyXh67oekslgbxy+B983sYTN7GPgT8PPElSWS3ZT9kFQWa3L8OTMrBS6MzBrh\n7h8mriyR7Kbnfkgqi/l0U6RRqFmIJMnwHnlqFJKSYj1VJSIiAmiAWyRjKDAoyaLGIZIBagODtdmP\n2sAgoOYhLU6nqkQygAKDkkxqHCIZQIFBSSY1DpEMoMCgJFMgjcPMppjZR2a2OvKAqNwGlhtsZuvN\nbKOZjU92nSLpQoFBSaagjjjeAgrcPQR8DNx3+AJm1gr4DXAJ0A24xsy6JbVKkTShh0VJMgVyVZW7\nL6wz+QHh27QfrhewMfJ8c8xsFjAMhRBF6qXAoCRLKlyOezPwcj3z84DP6kxXAucmpSKRLKHshzRF\nwhqHmS0CTq3nrQnu/npkmQlADTCzBbY3GhgN0Llz5+auTiTjKfshTZWwxuHuFzX2vpndCAwFBnj9\nDz6vAk6rM50fmdfQ9qYD0yH8zPF46xXJNo1lP9Q4pDFBXVU1GLgXuNzd9zaw2HLgbDPrYmZHA1cD\nc5NVo0imU/ZDmiqoq6p+DbQF3jKzcjN7EsDMOpnZfAB3rwHGAAuAdcAr7l4RUL0iGUfZD2mqoK6q\nOquB+VuBIXWm5wPzk1WXSDYZO6jrIWMcoOyHxCYVrqoSkQDoYVHSVGocIllM2Q9pCjUOEYmLsh+i\nxiEiMVP2Q0B3xxWROOi5HwJqHCISB2U/BNQ4RCQOyn4IqHGISBz03A8BDY6LSByU/RBQ4xCROCn7\nIWocIpJQyn1kHjUOEUkY5T4ykwbHRSRhlPvITGocIpIwyn1kJjUOEUkY5T4ykxqHiCSMch+ZSYPj\nIpIwyn1kJjUOEUko5T4yjxqHiKQcZT9SmxqHiKQUZT9SnwbHRSSlKPuR+tQ4RCSlKPuR+gJpHGY2\nxcw+MrPVZjbHzHIbWG6Lma0xs3IzK012nSKSfMp+pL6gjjjeAgrcPQR8DNzXyLLfdfdidy9JTmki\nEiRlP1JfII3D3Re6e01k8gMgP4g6RCT1DO+Rx6QRheTl5mBAXm4Ok0YUamA8hZi7B1uA2f8AL7v7\nC/W89wmwCzgATHP36Y2sZzQwGqBz5849P/300wRVLCKSecysLNYzOwm7HNfMFgGn1vPWBHd/PbLM\nBKAGmNnAavq5e5WZnQK8ZWYfufuS+haMNJXpACUlJcF2QxGRDJawxuHuFzX2vpndCAwFBngDhz3u\nXhX5vd3M5gC9gHobh4hkN4UGkyeoq6oGA/cCl7v73gaWOc7M2ta+Bi4G1iavShFJF7WhwarqL3EO\nhgZfW1kVdGkZKairqn4NtCV8+qnczJ4EMLNOZjY/skwHYKmZrQKWAfPc/c1gyhWRVKbQYHIFcssR\ndz+rgflbgSGR15uBomTWJSLpSaHB5FJyXETSnkKDyaXGISJpT6HB5NLdcUUk7emBUcmlxiEiGUEP\njEoeNQ4RyUrKfTSdGoeIZB09LKp5NDguIllHuY/mUeMQkayj3EfzqHGISNZR7qN51DhEJOso99E8\nGhwXkayj3EfzqHGISFZS7qPp1DhERGKk7EeYGoeISAyU/ThIg+MiIjFQ9uMgNQ4RkRgo+3GQGoeI\nSAyU/ThIjUNEJAbKfhykwXERkRgo+3GQGoeISIyU/QhT4xARSaBMzH4EMsZhZg+b2WozKzezhWbW\nqYHlBpvZejPbaGbjk12niEhz1GY/qqq/xDmY/XhtZVXQpTVLUIPjU9w95O7FwBvA/YcvYGatgN8A\nlwDdgGvMrFtyyxQRabpMzX4E0jjc/Z91Jo8DvJ7FegEb3X2zu38FzAKGJaM+EZGWkKnZj8AuxzWz\nR8zsM+Ba6jniAPKAz+pMV0bmNbS+0WZWamalO3bsaNliRUSaIFOzHwlrHGa2yMzW1vMzDMDdJ7j7\nacBMYExzt+fu0929xN1L2rdv39zViYg0W6ZmPxJ2VZW7XxTjojOB+cBPD5tfBZxWZzo/Mk9EJC1k\navYjkMtxzexsd98QmRwGfFTPYsuBs82sC+GGcTUwKkklioi0iEzMfgSV45hsZl2Br4FPgdsBIpfl\nPuXuQ9y9xszGAAuAVsAz7l4RUL0iIhIRSONw9/9sYP5WYEid6fmET2OJiGSFdAgMKjkuIpIi0uVh\nUbo7rohIikiXwKAah4hIikiXwKAah4hIikiXwKAah4hIikiXwKAGx0VEUkS6BAbVOEREUkg6BAbV\nOERE0lyysx9qHCIiaSyI7IcGx0VE0lgQ2Q81DhGRNBZE9kONQ0QkjQWR/VDjEBFJY0FkPzQ4LiKS\nxoLIfqhxiIikuWRnP3SqSkRE4qLGISIicVHjEBGRuKhxiIhIXNQ4REQkLubuQdfQ4sxsB/BpEz9+\nMvD3FiwnnWTzvkN277/2PXvV7v/p7t4+lg9kZONoDjMrdfeSoOsIQjbvO2T3/mvfs3PfoWn7r1NV\nIiISFzUOERGJixrHv5sedAEByuZ9h+zef+179op7/zXGISIicdERh4iIxEWNQ0RE4qLGEWFmg81s\nvZltNLPxQdeTbGa2xczWmFm5mZUGXU8imdkzZrbdzNbWmdfOzN4ysw2R3ycGWWMiNbD/D5hZVeT7\nLzezIUHWmChmdpqZvWNmH5pZhZndFZmf8d9/I/se93evMQ7AzFoBHwMDgUpgOXCNu38YaGFJZGZb\ngBJ3z/gglJn9B7AHeM7dCyLzfg587u6TI/9wONHdxwVZZ6I0sP8PAHvc/RdB1pZoZtYR6OjuK8ys\nLVAGDAduJMO//0b2/XvE+d3riCOsF7DR3Te7+1fALGBYwDVJgrj7EuDzw2YPA2ZEXs8g/D9URmpg\n/7OCu29z9xWR17uBdUAeWfD9N7LvcVPjCMsDPqszXUkT/0DTmAOLzKzMzEYHXUwAOrj7tsjrvwId\ngiwmIHeY2erIqayMO1VzODM7A+gB/Jks+/4P23eI87tX45Ba/dy9GLgE+FHkdEZW8vD522w7h/vf\nwJlAMbAN+GWw5SSWmR0P/B64293/Wfe9TP/+69n3uL97NY6wKuC0OtP5kXlZw92rIr+3A3MIn77L\nJn+LnAOuPRe8PeB6ksrd/+buB9z9a+D/kcHfv5kdRfgvzpnu/mpkdlZ8//Xte1O+ezWOsOXA2WbW\nxcyOBq4G5gZcU9KY2XGRwTLM7DjgYmBt45/KOHOBGyKvbwBeD7CWpKv9SzPiCjL0+zczA54G1rn7\n1DpvZfz339C+N+W711VVEZFL0B4DWgHPuPsjAZeUNGZ2JuGjDIDWwIuZvP9m9hJwAeHbSf8N+Cnw\nGvAK0JnwLfm/5+4ZOYDcwP5fQPhUhQNbgNvqnPPPGGbWD3gPWAN8HZn9E8Ln+jP6+29k368hzu9e\njUNEROKiU1UiIhIXNQ4REYmLGoeIiMRFjUNEROKixiEiInFR4xBJIWZ2gZm9EXQdIo1R4xARkbio\ncYg0gZldZ2bLIs8vmGZmrcxsj5k9GnnWwWIzax9ZttjMPojcRG5O7U3kzOwsM1tkZqvMbIWZfTOy\n+uPNbLaZfWRmMyOJX5GUocYhEicz+zZwFdA3cmPIA8C1wHFAqbt3B94lnMgGeA4Y5+4hwqnd2vkz\ngd+4exHQFw45AAABK0lEQVTQh/AN5iB819K7gW6Ebz7XN+E7JRKH1kEXIJKGBgA9geWRg4EcwjfF\n+xp4ObLMC8CrZnYCkOvu70bmzwB+F7k3WJ67zwFw930AkfUtc/fKyHQ5cAawNPG7JRIbNQ6R+Bkw\nw93vO2Sm2f89bLmm3s/nX3VeH0D/n0qK0akqkfgtBkaa2SkQfV716YT/fxoZWWYUsNTddwH/MLP+\nkfnfB96NPIGt0syGR9ZxjJkdm9S9EGki/UtGJE7u/qGZTQQWmtk3gP3Aj4AvgF6R97YTHgeB8G26\nn4w0hs3ATZH53wemmdlDkXVcmcTdEGky3R1XpIWY2R53Pz7oOkQSTaeqREQkLjriEBGRuOiIQ0RE\n4qLGISIicVHjEBGRuKhxiIhIXNQ4REQkLv8f2e1vENPmpRIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7feaafbaeac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting\n",
    "plt.plot(range(len(loss_history)), loss_history, 'o', label='Logistic Regression Training phase')\n",
    "plt.ylabel('cost')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Your Turn\n",
    "\n",
    "Please switch to the **Theano** backend and **restart** the notebook.\n",
    "\n",
    "You _should_ see no difference in the execution!\n",
    "\n",
    "**Reminder**: please keep in mind that you *can* execute shell commands from a notebook (pre-pending a `!` sign).\n",
    "Thus:\n",
    "\n",
    "```shell\n",
    "    !cat ~/.keras/keras.json\n",
    "```\n",
    "should show you the content of your keras configuration file."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Moreover\n",
    "\n",
    "Try to play a bit with the **learning reate** parameter to see how the loss history floats... "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise: Linear Regression\n",
    "To get familiar with automatic differentiation, we start by learning a simple linear regression model using Stochastic Gradient Descent (SGD).\n",
    "\n",
    "Recall that given a dataset $\\{(x_i, y_i)\\}_{i=0}^N$, with $x_i, y_i \\in \\mathbb{R}$, the objective of linear regression is to find two scalars $w$ and $b$ such that $y = w\\cdot x + b$ fits the dataset. In this tutorial we will learn $w$ and $b$ using SGD and a Mean Square Error (MSE) loss:\n",
    "\n",
    "$$\\mathcal{l} = \\frac{1}{N} \\sum_{i=0}^N (w\\cdot x_i + b - y_i)^2$$\n",
    "\n",
    "Starting from random values, parameters $w$ and $b$ will be updated at each iteration via the following rule:\n",
    "\n",
    "$$w_t = w_{t-1} - \\eta \\frac{\\partial \\mathcal{l}}{\\partial w}$$\n",
    "<br>\n",
    "$$b_t = b_{t-1} - \\eta \\frac{\\partial \\mathcal{l}}{\\partial b}$$\n",
    "\n",
    "where $\\eta$ is the learning rate.\n",
    "\n",
    "**NOTE:** Recall that **linear regression** is indeed a **simple neuron** with a linear activation function!!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Definition: Placeholders and Variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First of all, we define the necessary variables and placeholders for our computational graph. Variables maintain state across executions of the computational graph, while placeholders are ways to feed the graph with external data.\n",
    "\n",
    "For the linear regression example, we need three variables: `w`, `b`, and the learning rate for SGD, `lr`. \n",
    "\n",
    "Two placeholders `x` and `target` are created to store $x_i$ and $y_i$ values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Placeholders and variables\n",
    "x = K.placeholder()\n",
    "target = K.placeholder()\n",
    "w = K.variable(np.random.rand())\n",
    "b = K.variable(np.random.rand())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Notes:\n",
    "\n",
    "In case you're wondering what's the difference between a **placeholder** and a **variable**, in short:\n",
    "\n",
    "* Use `K.variable()` for trainable variables such as weights (`W`) and biases (`b`) for your model.\n",
    "* Use `K.placeholder()` to feed actual data (e.g. training examples)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Model definition\n",
    "Now we can define the $y = w\\cdot x + b$ relation as well as the MSE loss in the computational graph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define model and loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %load solutions/sol11.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, given the gradient of MSE wrt to `w` and `b`, we can define how we update the parameters via SGD:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %load solutions/sol12.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The whole model can be encapsulated in a `function`, which takes as input `x` and `target`, returns the current loss value and updates its parameter according to `updates`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = K.function(inputs=[x, target], outputs=[loss], updates=updates)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training\n",
    "Training is now just a matter of calling the `function` we have just defined. Each time `train` is called, indeed, `w` and `b` will be updated using the SGD rule.\n",
    "\n",
    "Having generated some random training data, we will feed the `train` function for several epochs and observe the values of `w`, `b`, and loss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Generate data\n",
    "np_x = np.random.rand(1000)\n",
    "np_target = 0.96*np_x + 0.24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Training\n",
    "loss_history = []\n",
    "for epoch in range(200):\n",
    "    current_loss = train([np_x, np_target])[0]\n",
    "    loss_history.append(current_loss)\n",
    "    if epoch % 20 == 0:\n",
    "        print(\"Loss: %.03f, w, b: [%.02f, %.02f]\" % (current_loss, K.eval(w), K.eval(b)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also plot the loss history:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Plot loss history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %load solutions/sol13.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Note:\n",
    "\n",
    "Please switch back your backend to `tensorflow` before moving on. It may be useful for next notebooks !-)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
